{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from functools import partial\n",
    "from sklearn.externals import joblib\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "sys.path.append('../')\n",
    "from src.utils import parallel_apply\n",
    "from src.feature_extraction import add_features_in_group\n",
    "\n",
    "DIR = '/PATH/TO/YOUR/DATA'\n",
    "description = pd.read_csv(os.path.join(DIR,'data/HomeCredit_columns_description.csv'),encoding = 'latin1')\n",
    "application = pd.read_csv(os.path.join(DIR, 'files/unzipped_data/application_train.csv'))\n",
    "bureau = pd.read_csv(os.path.join(DIR, 'files/unzipped_data/bureau.csv'))\n",
    "bureau_balance = pd.read_csv(os.path.join(DIR, 'files/unzipped_data/bureau_balance.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bureau_balance.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bureau_balance = bureau_balance.merge(bureau[['SK_ID_CURR', 'SK_ID_BUREAU']], on='SK_ID_BUREAU', how='right')\n",
    "bureau_balance.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bureau_balance.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Solution 5\n",
    "### Hand crafted features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _status_to_int(status):\n",
    "    if status in ['X', 'C']:\n",
    "        return 0\n",
    "    if pd.isnull(status):\n",
    "        return np.nan\n",
    "    return int(status)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bureau_balance['bureau_balance_dpd_level'] = bureau_balance['STATUS'].apply(_status_to_int)\n",
    "bureau_balance['bureau_balance_status_unknown'] = (bureau_balance['STATUS'] == 'X').astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bureau_balance.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "groupby = bureau_balance.groupby(['SK_ID_CURR'])\n",
    "features = pd.DataFrame({'SK_ID_CURR': bureau_balance['SK_ID_CURR'].unique()})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def last_k_installment_features(gr, periods):\n",
    "    gr_ = gr.copy()\n",
    "\n",
    "    features = {}\n",
    "    for period in periods:\n",
    "        if period > 10e10:\n",
    "            period_name = 'all_installment_'\n",
    "            gr_period = gr_.copy()\n",
    "        else:\n",
    "            period_name = 'last_{}_'.format(period)\n",
    "            gr_period = gr_[gr_['MONTHS_BALANCE'] >= (-1) * period]\n",
    "\n",
    "        features = add_features_in_group(features, gr_period, 'bureau_balance_dpd_level',\n",
    "                                             ['sum', 'mean', 'max', 'std', 'skew', 'kurt'],\n",
    "                                             period_name)\n",
    "        features = add_features_in_group(features, gr_period, 'bureau_balance_status_unknown',\n",
    "                                             ['sum', 'mean'],\n",
    "                                             period_name)\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "func = partial(last_k_installment_features, periods=[6, 12, 24, 60, 10e16])\n",
    "g = parallel_apply(groupby, func, index_name='SK_ID_CURR', num_workers=10, chunk_size=10000).reset_index()\n",
    "features = features.merge(g, on='SK_ID_CURR', how='left')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Trend features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trend_in_last_k_installment_features(gr, periods):\n",
    "    gr_ = gr.copy()\n",
    "    gr_.sort_values(['MONTHS_BALANCE'], ascending=False, inplace=True)\n",
    "\n",
    "    features = {}\n",
    "    for period in periods:\n",
    "        gr_period = gr_[gr_['MONTHS_BALANCE'] >= (-1) * period]\n",
    "\n",
    "        features = add_trend_feature(features, gr_period,\n",
    "                                         'bureau_balance_dpd_level', '{}_period_trend_'.format(period)\n",
    "                                         )\n",
    "    return features\n",
    "\n",
    "def add_trend_feature(features, gr, feature_name, prefix):\n",
    "    y = gr[feature_name].values\n",
    "    try:\n",
    "        x = np.arange(0, len(y)).reshape(-1, 1)\n",
    "        lr = LinearRegression()\n",
    "        lr.fit(x, y)\n",
    "        trend = lr.coef_[0]\n",
    "    except:\n",
    "        trend = np.nan\n",
    "    features['{}{}'.format(prefix, feature_name)] = trend\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "func = partial(trend_in_last_k_installment_features, periods=[6,12,24,60])\n",
    "g = parallel_apply(groupby, func, index_name='SK_ID_CURR', num_workers=10, chunk_size=10000).reset_index()\n",
    "features = features.merge(g, on='SK_ID_CURR', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def last_k_instalment_fractions(old_features, fraction_periods):\n",
    "    features = old_features[['SK_ID_CURR']].copy()\n",
    "    \n",
    "    for short_period, long_period in fraction_periods:\n",
    "        short_feature_names = _get_feature_names(old_features, short_period)\n",
    "        long_feature_names = _get_feature_names(old_features, long_period)\n",
    "        \n",
    "        for short_feature, long_feature in zip(short_feature_names, long_feature_names):\n",
    "            old_name_chunk = '_{}_'.format(short_period)\n",
    "            new_name_chunk ='_{}by{}_fraction_'.format(short_period, long_period)\n",
    "            fraction_feature_name = short_feature.replace(old_name_chunk, new_name_chunk)\n",
    "            features[fraction_feature_name] = old_features[short_feature]/old_features[long_feature]\n",
    "    return pd.DataFrame(features).fillna(0.0)\n",
    "\n",
    "def _get_feature_names(features, period):\n",
    "    return sorted([feat for feat in features.keys() if '_{}_'.format(period) in feat])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "g = last_k_instalment_fractions(features, fraction_periods=[(6, 12), (6, 24), (12,24), (12, 60)])\n",
    "features = features.merge(g, on='SK_ID_CURR', how='left')\n",
    "\n",
    "display(features.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "X = application.merge(features, on='SK_ID_CURR',how='left')[features.columns.tolist() + ['TARGET']]\n",
    "X_corr = abs(X.corr())\n",
    "X_corr.sort_values('TARGET', ascending=False)['TARGET']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "regex = '_bureau_balance'\n",
    "X_corr_truncated = X_corr.sort_values('TARGET', ascending=False).filter(regex=regex, axis=0)\n",
    "X_corr_truncated['TARGET']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "colnames = X_corr_truncated.index.tolist() + ['TARGET']\n",
    "plt.figure(figsize=(15, 15));\n",
    "sns.heatmap(X_corr_truncated[colnames], \n",
    "            xticklabels=colnames,\n",
    "            yticklabels=colnames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "X_to_plot = X[X.columns[0:20].tolist()+['TARGET']]\n",
    "X_corr_to_plot = abs(X_to_plot.corr())\n",
    "X_corr_to_plot.sort_values('TARGET', ascending=False)['TARGET']\n",
    "sns.heatmap(X_corr_to_plot, \n",
    "            xticklabels=X_corr_to_plot.columns,\n",
    "            yticklabels=X_corr_to_plot.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "columns_to_use = X.columns[20:40]\n",
    "\n",
    "for col in columns_to_use:\n",
    "    print(col)\n",
    "    X[col].hist(bins=50).plot()\n",
    "    plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
